﻿/*
 * Microsoft .NET  2.0+ version of Sphinx searchd client (Sphinx API)
 * 
 * Copyright (c) 2009, Shaun Berryman
 * All rights reserved
 * 
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License. You should have
 * received a copy of the GPL license along with this program; if you
 * did not, you can find it at http://www.gnu.org/
 */

using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Text;
using System.IO;
using System.Net;
using System.Net.Sockets;
using System.Diagnostics;

using SphinxSearchApi.Model;
using SphinxSearchApi.Extension;

namespace SphinxSearchApi
{
    public class SphinxClient
    {
        #region ReadWrite Variables

        private int _ReceiveTimeout;

        /// <summary>
        /// Get or Set the receive timeout for the TcpClient. When using a persistent connection, the 
        /// corresponding property for the tcpclient will be set.
        /// </summary>
        public int ReceiveTimeout
        {
            get { return _ReceiveTimeout; }
            set
            {
                _ReceiveTimeout = value;

                // Set the timeout on our socket if it exists
                if (_Socket != null)
                    _Socket.ReceiveTimeout = value;
            }
        }

        // Matches
        public MatchEnum MatchMode { get; set; }
        
        private int _Limit;
        /// <summary>
        /// Amount of matches to retrieve from result set, default is 20;
        /// </summary>
        public int Limit 
        { 
            get { return _Limit; }
            set
            {
                if (value < 1)
                    throw new ArgumentOutOfRangeException("Limit", value, "Match must be greater than 0 (zero)");

                _Limit = value;
            }
        }
        
        private int _Offset;
        /// <summary>
        /// Offset into result set, default is 0
        /// </summary>
        public int Offset 
        {
            get { return _Offset; }
            set
            {
                if (value < 0)
                    throw new ArgumentOutOfRangeException("Offset", value, "Offset must be greater than or equal to 0 (zero)");

                _Offset = value;
            }
        }
        
        private int _MaxMatches;
        /// <summary>
        /// Per-query max matches value
        /// </summary>
        public int MaxMatches 
        { 
            get { return _MaxMatches; }
            set
            {
                if (value < 1)
                    throw new ArgumentOutOfRangeException("MaxMatches", value, "Max matches must be greater than 0 (zero)");

                _MaxMatches = value;
            }
        }

        // Sort
        public SortEnum Sort { get; set; }
        public string SortBy { get; set; }

        
        /// <summary>
        /// Min document Id to match
        /// </summary>
        public int MinId { get; set; }

        /// <summary>
        /// Max document Id to match
        /// </summary>
        public int MaxId { get; set; }

        public GroupByEnum GroupByMode { get; set; }
        public string GroupBy { get; set; }
        public string GroupSort { get; set; }
        public string GroupDistinct { get; set; }

        private int _Cutoff;
        /// <summary>
        /// Max found matches threshold
        /// </summary>
        public int Cutoff 
        { 
            get { return _Cutoff; }
            set
            {
                if (value < 0)
                    throw new ArgumentOutOfRangeException("Cutoff", value, "Must be greater than or equal to 0 (zero)");

                _Cutoff = value;
            }
        }

        /// <summary>
        /// Distributed retries count
        /// </summary>
        public int RetryCount { get; set; }

        /// <summary>
        /// Distributed retry delay, milliseconds
        /// </summary>
        public int RetryDelay { get; set; }

        /// <summary>
        /// Set maximum query time, in milliseconds, per-index, 0 means "do not limit"
        /// </summary>
        public int MaxQueryTime { get; set; }

        public RankEnum Ranker { get; set; }

        private string _Select;
        /// <summary>
        /// Sets the select clause, listing specific attributes to fetch, and expressions to compute and fetch
        /// </summary>
        public string Select 
        {
            get { return _Select; }
            set
            {
                if (string.IsNullOrEmpty(value))
                    throw new ArgumentNullException("Select", "Select must be be specified");

                _Select = value;
            }
        }

        /// <summary>
        /// Sets per-index weights. Default is empty which disables weight summing.
        /// 
        /// Key=index name, Value=Weight
        /// </summary>
        public Dictionary<string, int> IndexWeights { get; set; }

        /// <summary>
        /// per-field weighted phrase ranks are computed as a product of LCS multiplied 
        /// by per-field weight specified by user. Per-field weights are 
        /// integer, default to 1, and can not be set lower than 1.
        /// 
        /// Key=Field name, Value=Weight.
        /// </summary>
        public Dictionary<string, int> FieldWeights { get; set; }

        /// <summary>
        /// Sets anchor point for and geosphere distance (geodistance) calculations, and enable them. 
        /// </summary>
        public AnchorPoint GeoAnchor { get; set; }

        #endregion
        
        #region ReadOnly Variables

        public string Error { get; private set; }
        public string Warning { get; private set; }

        public string Host { get; private set; }
        public int Port { get; private set; }

        #endregion

        #region Internal Variables

        internal readonly char[] _AttributeNameTrim = new char[] { '@' };

        private TcpClient _Socket;

        private int _FilterCount = 0;
        private MemoryStream _Filters;

        private List<byte[]> _RequestQueries;

        // Overrides
        private Dictionary<string, AttributeEnum> _OverrideTypes;
        private Dictionary<string, Dictionary<long, object>> _OverrideValues;

        #endregion

        #region Constructors

        /// <summary>
        /// Creates new SphinxClient instance.
        /// Host: localhost - Port: 9312
        /// </summary>
        public SphinxClient() : this("localhost", 9312) { }

        /// <summary>
        /// Creates new SphinxClient instance.
        /// Host: localhost - Port: 9312 - Timeout: 30000 (30 seconds)
        /// </summary>
        public SphinxClient(string host, int port) : this(host, port,  30000) {}

        /// <summary>
        /// Creates new SphinxClient instance.
        /// </summary>
        /// <param name="host">Hostname/IP address of the sphinx server</param>
        /// <param name="port">Port number of the instance. Default value is 9312</param>
        /// <param name="receiveTimeout">Timeout in milliseconds</param>
        public SphinxClient(string host, int port, int receiveTimeout)
        {
            // Check hostname
            if (string.IsNullOrEmpty(host))
                throw new ArgumentNullException("host", "Hostname or IP address required");

            // Check port number
            if (port < 0 || port >= 65536)
                throw new ArgumentOutOfRangeException("port", port, "Invalid port number. Valid port numbers range from 0 to 65535");

            // Set our properties
            this.Host = host;
            this.Port = port;
            this.ReceiveTimeout = receiveTimeout;

            // Set the defaults
            Offset = 0;
            Limit = 20;
            MatchMode = MatchEnum.All;
            MinId = 0;
            MaxId = 0;
            MaxMatches = 1000;
            Cutoff = 0;
            RetryCount = 0;
            RetryDelay = 0;
            ReceiveTimeout = 30000;
            Select = "*";
            Sort = SortEnum.Relevance;
            SortBy = "";

            _RequestQueries = new List<byte[]>();

            FieldWeights = new Dictionary<string, int>();
            IndexWeights = new Dictionary<string, int>();

            ResetFilters();
            ResetGroupBy();
            ResetOverrides();
        }

        #endregion

        #region Connection / Network

        /// <summary>
        /// Connect to searchd and exchange versions (internal method).
        /// </summary>
        /// <returns></returns>
        private TcpClient Connect()
        {
            // Return our existing socket if we've already created it
            // this will only be true on persistent connections
            if (_Socket != null)
                return _Socket;

            TcpClient sock = null;
            try
            {
                // Create the socket
                sock = new TcpClient(Host, Port);
                sock.ReceiveTimeout = ReceiveTimeout;

                // Get our network stream and reader/writer
                NetworkStream stream = sock.GetStream();
                var reader = new BinaryReader(stream);
                var writer = new BinaryWriter(stream);

                var version = reader.eReadInt32();
                if (version < 1)
                {
                    sock.Close();
                    Error = string.Concat("Expected searchd protocol version 1+, got version ", version);
                    return null;
                }

                // Send our version
                writer.WriteToStream((int)VersionEnum.MajorVersion);
            }
            catch (Exception e)
            {
                Error = string.Format(
                    "Connection to {0}:{1} failed: {2}",
                    Host,
                    Port,
                    e.Message
                    );

                try
                {
                    // Attempt to close the socket if it is open
                    if (sock != null)
                        sock.Close();
                }
                catch (IOException) 
                { 
                    // Silently ignore IO exceptions
                }

                return null;
            }

            // Return our socket
            return sock;
        }

        /// <summary>
        /// Open a persistent
        /// </summary>
        /// <returns></returns>
        public bool Open()
        {
            if (_Socket != null)
            {
                Error = "Already connected";
                return false;
            }

            // Attempt to connection
            var sock = Connect();

            // Make sure we connected
            if (sock == null)
                return false;

            // command, command version = 0, body length = 4, body = 1
            try
            {
                // Get our stream
                NetworkStream stream = sock.GetStream();
                var writer = new BinaryWriter(stream);

                writer.WriteToStream((short)CommandEnum.Persist);
                writer.WriteToStream((short)0);
                writer.WriteToStream(4);
                writer.WriteToStream(1);
            }
            catch (IOException e)
            {
                Error = string.Concat("Network error: ", e.ToString());
            }

            // Set our socket
            _Socket = sock;

            // Success!
            return true;
        }

        /// <summary>
        /// Close the persistent connection
        /// </summary>
        /// <returns></returns>
        public bool Close()
        {
            if (_Socket == null)
            {
                Error = "Not connected";
                return false;
            }

            try
            {
                _Socket.Close();
            }
            catch (IOException) 
            { 
                // Silently ignore IO exceptions
            }
            finally
            {
                _Socket = null;
            }

            return true;
        }

        private BinaryReader DoRequest(CommandEnum command, byte[] data)
        {
            // Connect
            var socket = Connect();
            if (socket == null)
                return null;

            // Send request
            try
            {
                var stream = socket.GetStream();
                var writer = new BinaryWriter(stream);

                // Command and version
                writer.WriteToStream(command);
                
                // Write our data
                writer.WriteToStream(data, true);
            }
            catch (Exception ex)
            {
                Error = string.Concat("Network error: ", ex.ToString());
                return null;
            }

            // Return the result
            return GetResponse(socket);
        }

        private BinaryReader GetResponse(TcpClient sock)
        {
            // connect
            NetworkStream stream = null;
            try
            {
                stream = sock.GetStream();
            }
            catch (IOException e)
            {
                throw new Exception(
                    string.Concat("GetInputStream() failed: ", e.Message)
                    );
            }

            // Read the result
            byte[] result = null;
            var reader = new BinaryReader(stream);
            StatusEnum status = 0;
            short version = 0;
            int length = 0;

            try
            {
                // Read status fields
                status = (StatusEnum)reader.eReadInt16();
                version = reader.eReadInt16();
                length = reader.eReadInt32();

                // Check to see if we have a zero-sized result
                if (length <= 0)
                    throw new Exception(
                        string.Format(
                            "Invalid result packet size (Length: {0})",
                            length.ToString()
                            )
                        );

                // Read ALL the bytes
                //result = new byte[length];
                //reader.Read(result, 0, length);
                result = reader.ReadBytes(length);
                

                // Check status
                if (status == StatusEnum.Warning)
                {
                    Error = "Searchd warning";
                    return null;
                }
                else if (status == StatusEnum.Error)
                {
                    Error = string.Concat("Searchd error: ", Encoding.UTF8.GetString(result, 4, result.Length - 4));
                    return null;
                }
                else if (status == StatusEnum.Retry)
                {
                    Error = string.Concat("Temporary searchd error: ", Encoding.UTF8.GetString(result, 4, result.Length - 4));
                    return null;
                }
                else if (status != StatusEnum.Ok)
                {
                    Error = string.Concat("Searched returned unknown status, code: ", status);
                    return null;
                }
            }
            catch (IOException e)
            {
                // build error message
                if (length != 0)
                    Error = string.Format(
                        "Failed to read searchd result: Status={0} (Code:{1}), Version={2}, Length={3}, Stack={4}",
                        status.ToString(),
                        (short)status,
                        version,
                        length,
                        e.StackTrace
                        );
                else
                    Error = string.Concat("Received zero-sized searchd result (searchd crashed?): ", e.Message);

                return null;
            }
            finally
            {
                // Free up the resources if we are NOT
                // using a persistent connection
                if (_Socket == null)
                {
                    try
                    {
                        if (reader != null)
                            reader.Close();

                        if (sock != null && !sock.Connected)
                            sock.Close();
                    }
                    catch (IOException)
                    {
                        // Silently ignore close failures; nothing could be done anyway
                    }
                }
            }

            // Get the result and create a memory stream from the result
            var ms = new MemoryStream(result);

            // Return a binary reader (as we typically use a reader NOT a byte array
            // to read the result
            return new BinaryReader(ms);
        }

        #endregion

        #region Methods

        /// <summary>
        /// Connects to searchd server, runs given search query with current settings, obtains and returns the result set.
        /// Index will default to null (all local indexes)
        /// </summary>
        /// <param name="query">Query string</param>
        /// <returns>Result of the query. Null on error</returns>
        public SearchResult Query(string query)
        {
            return Query(query, null, null);
        }

        /// <summary>
        /// Connects to searchd server, runs given search query with current settings, obtains and returns the result set.
        /// </summary>
        /// <param name="query">Query string</param>
        /// <param name="index">Default value for $index is "*" that means to query all local indexes. 
        /// Characters allowed in index names include Latin letters (a-z), numbers (0-9), minus 
        /// sign (-), and underscore (_); everything else is considered a separator.</param>
        /// <returns>Result of the query. Null on error</returns>
        public SearchResult Query(string query, string index)
        {
            return Query(query, index, null);
        }

        /// <summary>
        /// Connects to searchd server, runs given search query with current settings, obtains and returns the result set.
        /// </summary>
        /// <param name="query">Query string</param>
        /// <param name="index">Default value for $index is "*" that means to query all local indexes. 
        /// Characters allowed in index names include Latin letters (a-z), numbers (0-9), minus 
        /// sign (-), and underscore (_); everything else is considered a separator.</param>
        /// <param name="comment">Contents are sent to the query log, marked in square brackets, just before 
        /// the search terms, which can be very useful for debugging. Limited to 128 characters</param>
        /// <returns>Result of the query. Null on error</returns>
        public SearchResult Query(string query, string index, string comment)
        {
            if (!_RequestQueries.IsNullOrEmpty())
                throw new Exception("AddQuery() and Query() can not be combined; use RunQueries() instead");

            // Add the query
            AddQuery(query, index, comment);

            // Run our queries
            var results = RunQueries();
            if (results.IsNullOrEmpty())
                // Probably network error; error message should already be set
                return null; 

            // We only care about our initial result
            var result = results[0];
            if (result != null)
            {
                Warning = result.Warning;
                Error = result.Error;
            }

            // Return a null result if the result is null or if we had an error
            if (result == null || result.Status == StatusEnum.Error)
                return null;
            else
                return result;
        }

        /// <summary>
        /// Add to the query list, passing "*" as index without a comment
        /// </summary>
        /// <param name="query">query</param>
        /// <returns>-1 on failure, index on success</returns>
        public int AddQuery(string query)
        {
            return AddQuery(query, null, null);
        }

        /// <summary>
        /// Add to the query list
        /// </summary>
        /// <param name="query">query</param>
        /// <param name="index">Default value for $index is "*" that means to query all local indexes. 
        /// Characters allowed in index names include Latin letters (a-z), numbers (0-9), minus 
        /// sign (-), and underscore (_); everything else is considered a separator.</param>
        /// <returns>-1 on failure, index on success</returns>
        public int AddQuery(string query, string index)
        {
            return AddQuery(query, index, null);
        }

        /// <summary>
        /// Add to the query list
        /// </summary>
        /// <param name="query">query</param>
        /// <param name="index">Default value for $index is "*" that means to query all local indexes. 
        /// Characters allowed in index names include Latin letters (a-z), numbers (0-9), minus 
        /// sign (-), and underscore (_); everything else is considered a separator.</param>
        /// <param name="comment">Contents are sent to the query log, marked in square brackets, just before 
        /// the search terms, which can be very useful for debugging. Limited to 128 characters</param>
        /// <returns>-1 on failure, index on success</returns>
        public int AddQuery(string query, string index, string comment)
        {
            // Make sure we have a query
            //if (string.IsNullOrEmpty(query) && GeoAnchor == null)
            //    throw new ArgumentNullException("query", "Query is required. Do we really need to explain why?");

            // Override the index if null, setting to default value of all (*)
            if (string.IsNullOrEmpty(index))
                index = "*";

            // If we have a comment make sure it is not longer than 128 chars
            if (!string.IsNullOrEmpty(comment) && comment.Length > 128)
                comment = comment.Substring(0, 128);

            using (var stream = new MemoryStream())
            {
                using (var writer = new BinaryWriter(stream))
                {
                    try
                    {
                        writer.WriteToStream(Offset);
                        writer.WriteToStream(Limit);
                        writer.WriteToStream((int)MatchMode);
                        writer.WriteToStream((int)Ranker);

                        // Sort
                        writer.WriteToStream((int)Sort);
                        writer.WriteToStream(SortBy);

                        // Query
                        writer.WriteToStream(query);

                        // Weights (depreciated, use fieldWeights now)
                        writer.WriteToStream((int)0);

                        // Index
                        writer.WriteToStream(index);

                        // Range marker
                        writer.WriteToStream((int)0);

                        // Min and max ids
                        writer.WriteToStream(MinId);
                        writer.WriteToStream(MaxId);

                        // Filters
                        if (_Filters != null && _FilterCount > 0 && _Filters.Length > 0)
                        {
                            writer.WriteToStream(_FilterCount);
                            writer.WriteToStream(_Filters.ToArray(), false);
                        }
                        else
                            writer.WriteToStream((int)0);

                        // Group by, max mathces, sort-by
                        writer.WriteToStream((int)GroupByMode);
                        writer.WriteToStream(GroupBy);
                        writer.WriteToStream(Limit);
                        writer.WriteToStream(GroupSort);

                        // Cutoff and retry
                        writer.WriteToStream(Cutoff);
                        writer.WriteToStream(RetryCount);
                        writer.WriteToStream(RetryDelay);

                        // Group distinct
                        writer.WriteToStream(GroupDistinct);

                        // Anchor point
                        if (GeoAnchor != null)
                        {
                            writer.WriteToStream((int)1);
                            writer.WriteToStream(GeoAnchor.AttributeLatitude);
                            writer.WriteToStream(GeoAnchor.AttributeLongitude);
                            writer.WriteToStream((float)GeoAnchor.Latitude);
                            writer.WriteToStream((float)GeoAnchor.Longitude);
                        }
                        else
                            writer.WriteToStream((int)0);

                        // Per index weights
                        if (!IndexWeights.IsNullOrEmpty())
                        {
                            // Write the count
                            writer.WriteToStream(IndexWeights.Count);
                            foreach (var item in IndexWeights)
                            {
                                writer.WriteToStream(item.Key);
                                writer.WriteToStream(item.Value);
                            }
                        }
                        else
                            writer.WriteToStream((int)0);

                        // Max query time
                        writer.WriteToStream(MaxQueryTime);

                        // Per field weights
                        if (!FieldWeights.IsNullOrEmpty())
                        {
                            // Write the count
                            writer.WriteToStream(FieldWeights.Count);
                            foreach (var item in FieldWeights)
                            {
                                writer.WriteToStream(item.Key);
                                writer.WriteToStream(item.Value);
                            }
                        }
                        else
                            writer.WriteToStream((int)0);

                        // Comment (trim to 128 chars)
                        writer.WriteToStream(comment);

                        // Attributes override
                        if (!_OverrideTypes.IsNullOrEmpty() && !_OverrideValues.IsNullOrEmpty())
                        {
                            // Write our count
                            writer.WriteToStream(_OverrideTypes.Count);

                            // Add each override
                            foreach (var item in _OverrideTypes)
                            {
                                // Get our values for this key
                                var values = _OverrideValues[item.Key];

                                // Write the attribute name, value type and number of values
                                writer.WriteToStream(item.Key);
                                writer.WriteToStream((int)item.Value);
                                writer.WriteToStream(values.Count);

                                // Loop through each values
                                foreach (var valueItem in values)
                                {
                                    // Write the key
                                    writer.WriteToStream(valueItem.Key);

                                    switch (item.Value)
                                    {
                                        case AttributeEnum.Float: writer.WriteToStream((float)valueItem.Value); break;
                                        case AttributeEnum.BigInt: writer.WriteToStream((long)valueItem.Value); break;
                                        default: writer.WriteToStream((int)valueItem.Value); break;
                                    }
                                }
                            }
                        }
                        else
                            writer.WriteToStream((int)0);

                        // Select list
                        writer.WriteToStream(Select);

                        // All done
                        writer.Flush();

                        // Create the list if it doesnt exist
                        if (_RequestQueries == null)
                            _RequestQueries = new List<byte[]>();

                        // Add our bytes to the list
                        _RequestQueries.Add(stream.ToArray());

                        // Return our index
                        return _RequestQueries.Count - 1;
                    }
                    catch (Exception ex)
                    {
                        // Throw an exception
                        throw new Exception(
                            string.Concat("Error on adding query: ", ex.ToString())
                            );
                    }
                    finally
                    {
                        try
                        {
                            // Close the writer and base stream
                            writer.Close();
                        }
                        catch (Exception ex)
                        {
                            // Throw an exception
                            throw new Exception(
                                string.Concat("Error on adding query: ", ex.ToString())
                                );
                        }
                    }
                }
            }
        }

        public SearchResult[] RunQueries()
        {
            if (_RequestQueries.IsNullOrEmpty())
            {
                Error = "No queries defined, issue AddQuery() first";
                return null;
            }

            // Combine all of our request queries
            byte[] requestBuffer;
            using (var stream = new MemoryStream())
            {
                byte[] reqHeader = BitConverter.GetBytes(_RequestQueries.Count).Reverse();
                stream.Write(reqHeader, 0, reqHeader.Length);

                foreach (var item in _RequestQueries)
                    stream.Write(item, 0, item.Length);

                requestBuffer = stream.ToArray();
            }

            // Get our request count
            var requestCount = _RequestQueries.Count;

            // Get our result
            var reader = DoRequest(CommandEnum.Search, requestBuffer);
            if (reader == null)
                return new SearchResult[] 
                {
                    new SearchResult()
                    {
                        Error = Error
                    }.SetStatus((int)StatusEnum.Error)
                };
                //return null;

            // Query was sent ok; we can reset the requests now
            _RequestQueries.Clear();

            // Create our result array
            var resultCollection = new SearchResult[requestCount];

            try
            {
                // Loop through each request
                for (int i = 0; i < requestCount; i++)
                {
                    // Create a new result
                    var result = new SearchResult();

                    // Set our status
                    result.SetStatus(reader.eReadInt32());
                    if (result.Status != StatusEnum.Ok)
                    {
                        string message = reader.eReadUTF8();
                        if (result.Status == StatusEnum.Warning)
                            result.Warning = message;
                        else
                        {
                            // Set the error and move on
                            result.Error = message;
                            resultCollection[i] = result;
                            continue;
                        }
                    }

                    // Read the fields
                    var nFields = reader.eReadInt32();
                    result.Fields = new string[nFields];
                    for (int f = 0; f < nFields; f++)
                        result.Fields[f] = reader.eReadUTF8();

                    // Read the attributes
                    var nAttrib = reader.eReadInt32();
                    result.AttributeTypes = new AttributeEnum[nAttrib];
                    result.AttributeNames = new string[nAttrib];
                    for (int a = 0; a < nAttrib; a++)
                    {
                        result.AttributeNames[a] = reader.eReadUTF8();
                        result.AttributeTypes[a] = (AttributeEnum)reader.eReadInt32();
                    }

                    // Read the match count
                    var nMatch = reader.eReadInt32();
                    var id64 = reader.eReadInt32();
                    result.Matches = new Match[nMatch];
                    for (int m = 0; m < nMatch; m++)
                    {
                        var docInfo = new Match(
                            id64 == 0 ? reader.eReadUInt32() : reader.eReadInt64(),
                            reader.eReadInt32()
                            );

                        // Read our matches
                        for (int a = 0; a < result.AttributeTypes.Length; a++)
                        {
                            var name = result.AttributeNames[a];
                            var type = result.AttributeTypes[a];

                            if (type == AttributeEnum.BigInt)
                            {
                                docInfo.AttrValues.Add(reader.eReadUInt64());
                                continue;
                            }
                            else if (type == AttributeEnum.Float)
                            {
                                docInfo.AttrValues.Add(reader.eReadFloat());
                                continue;
                            }
                            else if (type == AttributeEnum.String)
                            {
                                docInfo.AttrValues.Add(reader.eReadUTF8());
                                continue;
                            }
                            else
                            {
                                var val = reader.eReadUInt32();
                                if ((type & AttributeEnum.MultiAttribute) != 0)
                                {
                                    var uintMulti = new UInt32[val];
                                    for (int k = 0; k < val; k++)
                                        uintMulti[k] = reader.eReadUInt32();

                                    docInfo.AttrValues.Add(uintMulti);
                                }
                                else
                                    docInfo.AttrValues.Add(val);
                            }
                        }

                        result.Matches[m] = docInfo;
                    }

                    // Get our metadata
                    result.Total = reader.eReadInt32();
                    result.TotalFound = reader.eReadInt32();
                    result.Time = reader.eReadInt32() / 1000.0F;

                    // Read the word info
                    result.Words = new WordInfo[reader.eReadInt32()];
                    for (int w = 0; w < result.Words.Length; w++)
                        result.Words[w] = new WordInfo(
                            reader.eReadUTF8(),
                            reader.eReadUInt32(),
                            reader.eReadUInt32()
                            );

                    // Add to the collection
                    resultCollection[i] = result;
                }
            }
            catch
            {
                Error = "Incomplete reply";
                return null;
            }

            // Return our collection
            return resultCollection;
        }

        /// <summary>
        /// Force attribute flush, and block until it completes.
        /// </summary>
        /// <returns>Current internal flush tag on success, -1 on failure.</returns>
        public int FlushAttributes()
        {
            var sphinxResult = DoRequest(CommandEnum.FlushAttributes, new byte[0]);
            if (sphinxResult == null)
                return -1;

            try
            {
                // Read our flush tag
                var iFlushTag = sphinxResult.eReadInt32();

                // Close the stream
                sphinxResult.Close();

                // Return the result
                return iFlushTag;
            }
            catch
            {
                Error = "Incomplete reply";
                return -1;
            }
        }

        /// <summary>
        /// Queries searchd status, and returns an array of status variable name and value pairs.
        /// </summary>
        /// <returns>Name value pair or status, null on failure</returns>
        public NameValueCollection Status()
        {
            // Connect
            var socket = Connect();
            if (socket == null)
                return null;

            // Create our result
            var result = new NameValueCollection();

            // Send request
            try
            {
                byte[] requestBuffer;
                using (var stream = new MemoryStream())
                {
                    using (var writer = new BinaryWriter(stream))
                    {
                        writer.WriteToStream((int)4);
                        writer.WriteToStream((int)1);

                        // Done
                        writer.Flush();
                    }

                    requestBuffer = stream.ToArray();
                }

                var reader = DoRequest(CommandEnum.Status, requestBuffer);
                if (reader == null)
                    return null;

                // Get our result
                var nRows = reader.eReadInt32();
                var nColumns = reader.eReadInt32();

                for (int i = 0; i < nRows; i++)
                {
                    // Read our name and value
                    var name = reader.eReadUTF8();
                    var value = reader.eReadUTF8();

                    // Add to result
                    result.Add(name, value);
                }

                // Close the reader
                reader.Close();

                // Return the result
                return result;
            }
            catch (Exception ex)
            {
                // network error, return null
                Error = string.Concat("Network error: ", ex.ToString());
                return null;
            }
        }

        /// <summary>
        /// Escape a string for sphinx.
        /// </summary>
        /// <param name="input">String to escape</param>
        /// <returns>An escaped string that is safe to send to sphinxd</returns>
        public string EscapeString(string input)
        {
            StringBuilder result = new StringBuilder();

            // Char to escape
            var replaceHash = new List<char>()
                {
                    '\\', '(',')','|','-','!','@','~','"','&', '/', '^', '$', '='
                };

            foreach (var item in input.ToCharArray())
            {
                if (replaceHash.Contains(item))
                    result.Append('\\' + item);
                else
                    result.Append(item);
            }

            return result.ToString();
        }

        /// <summary>
        /// Excerpts (snippets) builder function. Connects to searchd, asks it to generate
        /// excerpts (snippets) from given documents, and returns the results.
        /// </summary>
        /// <param name="docs">Plain array of strings that carry the documents' contents</param>
        /// <param name="index">Index to process. Different settings (such as charset, morphology, wordforms) from given index will be used</param>
        /// <param name="words">String that contains the keywords to highlight</param>
        /// <returns>Plain array of strings with excerpts (snippets) on success. Null on failure.</returns>
        public string[] BuildExcerpts(string[] docs, string index, string words)
        {
            return BuildExcerpts(docs, index, words, null);
        }

        /// <summary>
        /// Excerpts (snippets) builder function. Connects to searchd, asks it to generate
        /// excerpts (snippets) from given documents, and returns the results.
        /// </summary>
        /// <param name="docs">Plain array of strings that carry the documents' contents</param>
        /// <param name="index">Index to process. Different settings (such as charset, morphology, wordforms) from given index will be used</param>
        /// <param name="words">String that contains the keywords to highlight</param>
        /// <param name="options">Additional optional highlighting parameters</param>
        /// <returns>Plain array of strings with excerpts (snippets) on success. Null on failure.</returns>
        public string[] BuildExcerpts(string[] docs, string index, string words, ExcerptOptions options)
        {
            if (docs.IsNullOrEmpty())
                throw new ArgumentNullException("docs", "No documents to process");

            if (string.IsNullOrEmpty(index))
                throw new ArgumentNullException("index", "No index to process documents");

            if (string.IsNullOrEmpty(words))
                throw new ArgumentNullException("words", "No words to highlight");

            // Create our default options if nothing is specified
            if (options == null)
                options = new ExcerptOptions();

            // Build our request
            byte[] requestBuffer;
            try
            {
                using (var stream = new MemoryStream())
                {
                    using (var writer = new BinaryWriter(stream))
                    {
                        writer.WriteToStream((int)0);

                        // Set our flags
                        int iFlags = 1;
                        if (options.ExactPhrase) iFlags |= 2;
                        if (options.SinglePassage) iFlags |= 4;
                        if (options.WeightOrder) iFlags |= 16;
                        writer.WriteToStream(iFlags);

                        writer.WriteToStream(index);
                        writer.WriteToStream(words);

                        // Send options
                        writer.WriteToStream(options.BeforeMatch);
                        writer.WriteToStream(options.AfterMatch);
                        writer.WriteToStream(options.ChunkSeparator);
                        writer.WriteToStream(options.Limit);
                        writer.WriteToStream(options.Around);

                        // Send documents
                        writer.WriteToStream(docs.Length);
                        foreach (var item in docs)
                            writer.WriteToStream(item);

                        // Done
                        writer.Flush();
                    }

                    // Set our buffer
                    requestBuffer = stream.ToArray();
                }
            }
            catch (Exception ex)
            {
                Error = string.Concat("Internal error, failed to build request: ", ex.ToString());
                return null;
            }

            // Do our request
            var reader = DoRequest(CommandEnum.Excerpt, requestBuffer);
            if (reader == null)
                return null;

            // Parse the response
            try
            {
                var result = new string[docs.Length];
                for (int i = 0; i < docs.Length; i++)
                    result[i] = reader.eReadUTF8();

                // Done with the reader
                reader.Close();

                // Return our result
                return result;
            }
            catch 
            {
                Error = "Incomplete reply";
                return null;
            }
        }

        /// <summary>
        /// Instantly updates given attribute values in given documents
        /// </summary>
        /// <param name="index">Index to be updated</param>
        /// <param name="attributes">Array of attribute names</param>
        /// <param name="values">List of documentId(s) and the values to update.</param>
        /// <returns>Number of updated documents or -1 on failure</returns>
        public int UpdateAttributes(string index, string[] attributes, List<UpdateValue> values)
        {
            if (string.IsNullOrEmpty(index))
                throw new ArgumentNullException("index", "No index name provided");

            if (attributes.IsNullOrEmpty())
                throw new ArgumentNullException("attributes", "No attribute names provided");

            if (values.IsNullOrEmpty())
                throw new ArgumentNullException("values", "No update entries provided");

            // Check our values
            var attributeCount = attributes.Length + 1;
            foreach (var item in values)
            {
                if (item.Values.Count != attributeCount)
                    throw new ArgumentException(
                        string.Format(
                            "Update DocumentId: {0} has the wrong number of attributes {1}, should have {2}",
                            item.DocumentId.ToString(),
                            item.Values.Count.ToString("N0"),
                            attributeCount.ToString("N0")
                            )
                        );
            }

            // Build the request
            byte[] requestBuffer;
            try
            {
                using (var stream = new MemoryStream())
                {
                    using (var writer = new BinaryWriter(stream))
                    {
                        writer.WriteToStream(index);

                        // Write our attributes
                        foreach (var item in attributes)
                            writer.WriteToStream(item.TrimStart(_AttributeNameTrim));

                        // Write our value count
                        writer.WriteToStream(values.Count);
                        foreach (var item in values)
                        {
                            // Send the docid (int64)
                            writer.WriteToStream(item.DocumentId);

                            // Send each value
                            foreach (var val in item.Values)
                                writer.WriteToStream(val);
                        }

                        // All done
                        writer.Flush();
                    }

                    // Set the buffer
                    requestBuffer = stream.ToArray();
                }
            }
            catch (Exception ex)
            {
                Error = string.Concat("Internal error, failed to build request: ", ex.ToString());
                return -1;
            }

            // Get our response
            var reader = DoRequest(CommandEnum.Update, requestBuffer);
            if (reader == null)
                return -1;

            try
            {
                // Read in the result
                var result = reader.eReadInt32();

                // Close the stream
                reader.Close();

                // Return the result
                return result;
            }
            catch
            {
                Error = "Incomplete reply";
                return -1;
            }

        }

        /// <summary>
        /// Extracts keywords from query using tokenizer settings for given index, optionally with per-keyword occurrence statistics
        /// </summary>
        /// <param name="query">Query to extract keywords from</param>
        /// <param name="index">Name of index to get tokenizing settings and keyword occurrence statistics from</param>
        /// <param name="hits">Whether keyword statistics are required</param>
        public List<KeywordResult> BuildKeywords(string query, string index, bool hits)
        {
            if (string.IsNullOrEmpty(query))
                throw new ArgumentNullException("query", "No query provided");

            if (string.IsNullOrEmpty(index))
                throw new ArgumentNullException("index", "No index provided");

            byte[] requestBuffer;
            try
            {
                using (var stream = new MemoryStream())
                {
                    using (var writer = new BinaryWriter(stream))
                    {
                        writer.WriteToStream(query);
                        writer.WriteToStream(index);
                        writer.WriteToStream(hits ? 1 : 0);

                        // Done
                        writer.Flush();
                    }

                    // Set the buffer
                    requestBuffer = stream.ToArray();
                }
            }
            catch (Exception ex)
            {
                Error = string.Concat("Internal error, failed to build request: ", ex.ToString());
                return null;
            }

            // Get our response
            var reader = DoRequest(CommandEnum.Keywords, requestBuffer);
            if (reader == null)
                return null;

            try
            {
                var nWords = reader.eReadInt32();
                var result = new List<KeywordResult>(nWords);

                for (int i = 0; i < nWords; i++)
                {
                    var item = new KeywordResult();
                    item.Tokenized = reader.eReadUTF8();
                    item.Normalized = reader.eReadUTF8();

                    if (hits)
                    {
                        item.Documents = reader.eReadInt64();
                        item.Hits = reader.eReadInt64();
                    }

                    // Add to the list
                    result.Add(item);
                }

                // Close the stream
                reader.Close();

                // Return the result
                return result;
            }
            catch
            {
                Error = "Incomplete response";
                return null;
            }
        }

        #endregion

        #region Reset and Setters

        /// <summary>
        /// Sets temporary (per-query) per-document attribute value overrides. Only supports scalar 
        /// attributes. Values must be a hash that maps document IDs to overridden attribute values.
        /// </summary>
        /// <param name="attributeName"></param>
        /// <param name="attributeType"></param>
        /// <param name="values"></param>
        public void SetOverride(string attributeName, AttributeEnum attributeType, Dictionary<long, object> values)
        {
            // Check our name
            if (string.IsNullOrEmpty(attributeName))
                throw new ArgumentNullException("attributeName", "Attribute name must be provided");

            // Make sure it is a valid override type
            if (attributeType == AttributeEnum.None ||
                attributeType == AttributeEnum.Ordinal ||
                attributeType == AttributeEnum.String ||
                attributeType == AttributeEnum.MultiAttribute)
                throw new ArgumentOutOfRangeException(
                    "attributeType",
                    attributeType.ToString(),
                    "Unsupported attribute type, must be one of the following: Integer, Timestamp, Boolean, Float or BigInt");

            _OverrideTypes.Add(attributeName.TrimStart(_AttributeNameTrim), attributeType);
            _OverrideValues.Add(attributeName.TrimStart(_AttributeNameTrim), values);
        }

        /// <summary>
        /// Reset all currently set filters (for multi-queries)
        /// </summary>
        public void ResetFilters()
        {
            // Close and dispose it first
            if (_Filters != null)
            {
                _Filters.Close();
                _Filters.Dispose();
            }

            // Create a new stream
            _Filters = new MemoryStream();

            // Reset the count
            _FilterCount = 0;

            // Reset the geo anchor
            GeoAnchor = null;
        }

        /// <summary>
        /// Clear groupby settings (for multi-queries).
        /// </summary>
        public void ResetGroupBy()
        {
            GroupBy = "";
            GroupByMode = GroupByEnum.Day;
            GroupSort = "@group desc";
            GroupDistinct = "";
        }

        /// <summary>
        /// Clear all attribute value overrides (for multi-queries)
        /// </summary>
        public void ResetOverrides()
        {
            if (_OverrideTypes == null)
                _OverrideTypes = new Dictionary<string, AttributeEnum>();
            else
                _OverrideTypes.Clear();

            if (_OverrideValues == null)
                _OverrideValues = new Dictionary<string, Dictionary<long, object>>();
            else
                _OverrideValues.Clear();
        }

        /// <summary>
        /// Add filter(s). Only match records where attribute value is in the given set
        /// </summary>
        /// <param name="attributeName">Name of attribute to filter on</param>
        /// <param name="value">Array of acceptable values</param>
        /// <param name="exclude"></param>
        public void SetFilter(string attributeName, int value, bool exclude)
        {
            SetFilter(attributeName, new int[] { value }, exclude);
        }

        /// <summary>
        /// Add filter(s). Only match records where attribute value is in the given set
        /// </summary>
        /// <param name="attributeName">Name of attribute to filter on</param>
        /// <param name="values">Array of acceptable values</param>
        /// <param name="exclude"></param>
        public void SetFilter(string attributeName, List<int> values, bool exclude)
        {
            SetFilter(attributeName, values.ToArray(), exclude);
        }

        /// <summary>
        /// Add filter(s). Only match records where attribute value is in the given set
        /// </summary>
        /// <param name="attributeName">Name of attribute to filter on</param>
        /// <param name="values">Array of acceptable values</param>
        /// <param name="exclude"></param>
        public void SetFilter(string attributeName, int[] values, bool exclude)
        {
            // Check our name
            if (string.IsNullOrEmpty(attributeName))
                throw new ArgumentNullException("attributeName", "Attribute name must be provided");
            
            if (values == null || values.Length < 1)
                throw new ArgumentNullException("values", "Values null or empty and are required to add a filter");

            _Filters.WriteToStream(attributeName.TrimStart(_AttributeNameTrim));
            _Filters.WriteToStream((int)FilterEnum.Values);

            for (int i = 0; i < values.Length; i++)
                _Filters.WriteToStream(values[i]);

            _Filters.WriteToStream(exclude ? 1 : 0);

            _FilterCount++;
        }

        /// <summary>
        /// Add filter(s). Only match records where attribute value is in the given set
        /// </summary>
        /// <param name="attributeName">Name of attribute to filter on</param>
        /// <param name="value">Array of acceptable values</param>
        /// <param name="exclude"></param>
        public void SetFilter(string attributeName, long value, bool exclude)
        {
            SetFilter(attributeName, new long[] { value }, exclude);
        }

        /// <summary>
        /// Add filter(s). Only match records where attribute value is in the given set
        /// </summary>
        /// <param name="attributeName">Name of attribute to filter on</param>
        /// <param name="values">Array of acceptable values</param>
        /// <param name="exclude"></param>
        public void SetFilter(string attributeName, List<long> values, bool exclude)
        {
            SetFilter(attributeName, values.ToArray(), exclude);
        }

        /// <summary>
        /// Add filter(s). Only match records where attribute value is in the given set
        /// </summary>
        /// <param name="attributeName">Name of attribute to filter on</param>
        /// <param name="values">Array of acceptable values</param>
        /// <param name="exclude"></param>
        public void SetFilter(string attributeName, long[] values, bool exclude)
        {
            // Check our name
            if (string.IsNullOrEmpty(attributeName))
                throw new ArgumentNullException("attributeName", "Attribute name must be provided");

            if (values == null || values.Length < 1)
                throw new ArgumentNullException("values", "Values null or empty and are required to add a filter");

            _Filters.WriteToStream(attributeName.TrimStart(_AttributeNameTrim));
            _Filters.WriteToStream((int)FilterEnum.Values);

            // Issue 2: Bug with SetFilter() function 
            // Add the lenth of the array
            _Filters.WriteToStream(values.Length);

            for (int i = 0; i < values.Length; i++)
                _Filters.WriteToStream(values[i]);

            _Filters.WriteToStream(exclude ? 1 : 0);

            _FilterCount++;
        }

        /// <summary>
        /// Set an integer range filter
        /// </summary>
        /// <param name="attributeName">Name of attribute to filter on</param>
        /// <param name="min">Minimum value of the range</param>
        /// <param name="max">Maximum value of the range</param>
        /// <param name="exclude"></param>
        public void SetFilterRange(string attributeName, int min, int max, bool exclude)
        {
            SetFilterRange(attributeName, (long)min, (long)max, exclude);
        }

        /// <summary>
        /// Set an integer range filter
        /// </summary>
        /// <param name="attributeName">Name of attribute to filter on</param>
        /// <param name="min">Minimum value of the range</param>
        /// <param name="max">Maximum value of the range</param>
        /// <param name="exclude"></param>
        public void SetFilterRange(string attributeName, long min, long max, bool exclude)
        {
            // Check our name
            if (string.IsNullOrEmpty(attributeName))
                throw new ArgumentNullException("attributeName", "Attribute name must be provided");

            if (min >= max)
                throw new ArgumentNullException("min/max", "Min must be less than or equal to max");

            _Filters.WriteToStream(attributeName.TrimStart(_AttributeNameTrim));
            _Filters.WriteToStream((int)FilterEnum.Range);
            _Filters.WriteToStream(min);
            _Filters.WriteToStream(max);
            _Filters.WriteToStream(exclude ? 1 : 0);

            _FilterCount++;
        }

        public void SetFilterFloatRange(string attributeName, float min, float max, bool exclude)
        {
            // Check our name
            if (string.IsNullOrEmpty(attributeName))
                throw new ArgumentNullException("attributeName", "Attribute name must be provided");

            if (min >= max)
                throw new ArgumentOutOfRangeException("min/max", string.Concat(min.ToString(), "/", max.ToString()), "Min must be less than or equal to max");

            //_Filters.WriteToStream(attributeName.TrimStart(_AttributeNameTrim));
            _Filters.WriteToStream(attributeName);
            _Filters.WriteToStream((int)FilterEnum.FloatRange);
            _Filters.WriteToStream(min);
            _Filters.WriteToStream(max);
            _Filters.WriteToStream(exclude ? 1 : 0);

            _FilterCount++;
        }

        #endregion
    }
}
